{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "已爬取第1页\n",
      "已爬取第2页\n"
     ]
    }
   ],
   "source": [
    "import requests as rqs\n",
    "import pandas as pd\n",
    "from bs4 import BeautifulSoup\n",
    "import time\n",
    "\n",
    "headers = {\"User-Agent\":\"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36\"}\n",
    "rst = []\n",
    "\n",
    "for i in range(1,2492):\n",
    "    try:\n",
    "        url = \"http://www.xinfadi.com.cn/marketanalysis/0/list/\"+str(i)+\".shtml\"\n",
    "        r = rqs.get(url,headers = headers)\n",
    "        soup = BeautifulSoup(r.text,\"lxml\")\n",
    "        data_list = soup.find_all(\"tr\",class_ = \"tr_color\")\n",
    "        for eachone in data_list:\n",
    "            name = eachone.contents[0].text\n",
    "            min_price = eachone.contents[1].text.strip()\n",
    "            mean_price = eachone.contents[2].text.strip()\n",
    "            max_price = eachone.contents[3].text.strip()\n",
    "            standard = eachone.contents[4].text\n",
    "            unit = eachone.contents[5].text\n",
    "            date = eachone.contents[6].text.strip()\n",
    "            list1 = [date,name,min_price,max_price,mean_price,standard,unit]\n",
    "            rst.append(list1)\n",
    "        time.sleep(0.2)\n",
    "        print(\"已爬取第%d页\"%i)\n",
    "    except:\n",
    "        print(\"第%d页有误\"%i)\n",
    "\n",
    "\n",
    "dataf = pd.DataFrame(rst,columns = [\"日期\",\"名称\",\"最低售价\",\"最高售价\",\"均价\",\"规格\",\"单位\"])\n",
    "local_time = time.strftime('%Y.%m.%d',time.localtime(time.time()))\n",
    "excel = local_time + \"-\"+\"新发地菜价\"\n",
    "file = \"C:\\\\Users\\\\fengy\\\\Desktop\\\\数据存储\\\\\"+excel+\".xlsx\"\n",
    "dataf.to_excel(file)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
